import requests
from lxml import etree
url='http://www.itangyuan.com/book/chapter/4689181/50034609.html'
headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'
    }
response=requests.get(url,headers=headers)
response.encoding=response.apparent_encoding
data=response.text
parse=etree.HTML(data)
hrefs=parse.xpath('//ul [@class="catalogue-list nano-content"]/li[position()>2]')
hrefs_data=[]
for i in hrefs:
    href_list=i.xpath('.//a/@href')
    for j in href_list:
        href='http://www.itangyuan.com'+j
        hrefs_data.append(href)
chapter_list=[]
info_list=[]
for url_list in hrefs_data:
    response1=requests.get(url_list,headers=headers)
    response1.encoding=response1.apparent_encoding
    parse1=etree.HTML(response1.text)
    chapters=parse1.xpath('//h1/text()')
    infos=parse1.xpath('//div[@class="section-main-con"]/p/text()')
    for chapter in chapters:
        chapter_list.append(chapter)
    for info in infos:
        info_list.append(info)
with open('小说','a',encoding='utf-8') as f:
    for x in chapter_list:
        f.write(x)
        for j in info_list:
                f.write(j)









